# # -*- coding: UTF-8 -*-
# """
# @author:zhangxing
# @file:ver2_spider_20201228.py
# @time:2020/12/28
# """
# import json
# import re
# import time
#
# import pypinyin
# import requests
# import scrapy
# import xlwt
# from lxml import etree
# from xlwt import Workbook
#
# from commonresources.spider_items.extra_demand.items_crawl_hang_ye_bai_qiang import convert_dict_HangYeCompanyCount, \
#     HangYeCompanyCount
# from spidertools.db_utils.mongdb_utils import get_mongo_connect
#
#
# class HangYeCompanyCountSpider(scrapy.Spider):
#     """
#
#     """
#     name = "HangYeCompanyCount"
#
#     # name_zh = ""
#     # province = ""
#     # allowed_domains = ['']
#     # start_urls = [""]
#
#     def __init__(self):
#         super(HangYeCompanyCountSpider, self).__init__()
#         self._, self.mongo_db = get_mongo_connect(mongo_url="127.0.0.1:27017")
#         self.mongo_table = self.mongo_db['project_html']
#         self.convert_dict = convert_dict_HangYeCompanyCount
#         self.huabei = ['天津市', '吉林省', '河北省', '北京市', '黑龙江省', '山东省', '辽宁省']
#         self.huadong = ['上海省', '江苏省', '浙江省', '安徽省']
#         self.huanan = ['海南省', '广东省', '广西省', '福建省']
#         self.huazhong = ['湖南省', '江西省', '湖北省', '河南省']
#         self.xibei = ['新疆省', '青海省', '宁夏省', '甘肃省', '内蒙古省', '山西省', '陕西省']
#         self.xinan = ['贵州省', '重庆省', '四川省', '云南省']
#
#     def fake_headers(self):
#         return {
#             # ":authority": "www.qcc.com",
#             # ":method": "GET",
#             # ":path": "/web/search?key=%E9%92%A2%E6%9E%84&filter=%7B%22rchain%22%3A%5B%7B%22pr%22%3A%22ZJ%22%7D%5D%7D",
#             # ":scheme": "https",
#             "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
#             "accept-encoding": "gzip, deflate, br",
#             "accept-language": "zh-CN,zh;q=0.9",
#             "cache-control": "max-age=0",
#             "cookie": "acw_tc=77a7ed9d16091191199457995edb7aba5682690532e869160bcff01df1; QCCSESSID=phk94k4k6oe8jfonk39qn5tdu0; zg_did=%7B%22did%22%3A%20%22176a6f93cd9151-0d84cd483c2ac6-3352487a-1fa400-176a6f93cda10b%22%7D; UM_distinctid=176a6f93ffd54c-07fd90d4c5a16c-3352487a-1fa400-176a6f93ffe9e5; CNZZDATA1254842228=1037414225-1609119118-https%253A%252F%252Fwww.baidu.com%252F%7C1609119118; hasShow=1; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201609119120604%2C%22updated%22%3A%201609119189078%2C%22info%22%3A%201609119120607%2C%22superProperty%22%3A%20%22%7B%5C%22%E5%BA%94%E7%94%A8%E5%90%8D%E7%A7%B0%5C%22%3A%20%5C%22%E4%BC%81%E6%9F%A5%E6%9F%A5%E7%BD%91%E7%AB%99%5C%22%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%5C%22%24utm_source%5C%22%3A%20%5C%22baidu1%5C%22%2C%5C%22%24utm_medium%5C%22%3A%20%5C%22cpc%5C%22%2C%5C%22%24utm_term%5C%22%3A%20%5C%22pzsy%5C%22%7D%22%2C%22referrerDomain%22%3A%20%22www.baidu.com%22%2C%22cuid%22%3A%20%22undefined%22%7D",
#             "dnt": "1",
#             "referer": "https://www.qcc.com/boss_search?key=%E9%92%A2%E6%9E%84",
#             "upgrade-insecure-requests": "1",
#             "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3823.400 QQBrowser/10.7.4307.400",
#         }
#
#     def fake_json_requests_headers(self, q):
#         return {
#             # ":authority": "www.qcc.com",
#             # ":method": "POST",
#             # ":path": "/api/search/searchMulti",
#             # ":scheme": "https",
#             # "Host": "www.qcc.com",
#             # "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
#             # "Accept": "application/json, text/plain, */*",
#             # "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
#             # "Accept-Encoding": "gzip, deflate, br",
#             # "Content-Type": "application/json;charset=utf-8",
#             # "X-Requested-With": "XMLHttpRequest",
#             # "Content-Length": "87",
#             # "Origin": "https://www.qcc.com",
#             # "Connection": "keep-alive",
#             # "Referer": f"https://www.qcc.com/web/search?key=%E9%92%A2%E6%9E%84&filter=%7B%22rchain%22%3A%5B%7B%22pr%22%3A%22GD%22%7D%5D%7D",
#             # "Cookie": f"zg_did=%7B%22did%22%3A%20%22175cebd98ba23f-071eed5a3264258-4c3f2678-1fa400-175cebd98bb67c%22%7D; "
#             #           f"zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201609124193492%2C%22updated%22%3A%20160912507"
#             #           f"9397%2C%22info%22%3A%201609124193498%2C%22superProperty%22%3A%20%22%7B%5C%22%E5%BA%94%E7%94%A8%E5%"
#             #           f"90%8D%E7%A7%B0%5C%22%3A%20%5C%22%E4%BC%81%E6%9F%A5%E6%9F%A5%E7%BD%91%E7%AB%99%5C%22%7D%22%2C%22plat"
#             #           f"form%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%"
#             #           f"22%3A%20%22undefined%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%7D; UM_distinctid=175cebd991d41-00aa2f1d9a"
#             #           f"a946-4c3f2678-1fa400-175cebd991e788; CNZZDATA1254842228=1706323772-1605488935-%7C1609123148;"
#             #           f" _uab_collina=160549133390296072120127; acw_tc=3ca9059f16091241906695817e4ef3d7247748325b0721e44fbeca9a19;"
#             #           f" hasShow=1; QCCSESSID={q}"
#             "Accept": "application/json, text/plain, */*",
#             "Accept-Encoding": "gzip, deflate, br",
#             "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
#             "Connection": "keep-alive",
#             # "Content-Length": "98",
#             "Content-Type": "application/json;charset=UTF-8",
#             "Cookie": "acw_tc=1bdd18a716091532492117139e207417b87fe9ddd7dedb5726834815e0; QCCSESSID=uergjhid5umkjav38q84jvfd57; zg_did=%7B%22did%22%3A%20%22176a90207129df-0564d2df74462c-5a301e44-1fa400-176a9020713831%22%7D; hasShow=1; _uab_collina=160915325244384004630792; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201609153251097%2C%22updated%22%3A%201609153273153%2C%22info%22%3A%201609153251102%2C%22superProperty%22%3A%20%22%7B%5C%22%E5%BA%94%E7%94%A8%E5%90%8D%E7%A7%B0%5C%22%3A%20%5C%22%E4%BC%81%E6%9F%A5%E6%9F%A5%E7%BD%91%E7%AB%99%5C%22%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%22%3A%20%22undefined%22%7D",
#             "Host": "www.qcc.com",
#             "Origin": "https://www.qcc.com",
#             # "Referer": "https://www.qcc.com/web/search?key=%E9%92%A2%E6%9E%84&filter=%7B%22rchain%22%3A%5B%7B%22pr%22%3A%22JS%22%7D%5D%7D",
#             "Sec-Fetch-Dest": "empty",
#             "Sec-Fetch-Mode": "cors",
#             "Sec-Fetch-Site": "same-origin",
#             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
#             "X-Requested-With": "XMLHttpRequest",
#         }
#
#     def get_district(self, pro_name):
#         if pro_name in self.huadong:
#             district = "华东"
#         elif pro_name in self.huabei:
#             district = "华北"
#         elif pro_name in self.huanan:
#             district = "华南"
#         elif pro_name in self.huazhong:
#             district = "华中"
#         elif pro_name in self.xibei:
#             district = "西北"
#         elif pro_name in self.xinan:
#             district = "西南"
#         else:
#             district = "unkown"
#         return district
#
#     def get_search_condition(self, provinces, response):
#         search_province_dict = []
#         for index, province in enumerate(provinces):
#             pattern = '"value":"(.{2,4})","count":\d*,"desc":"' + province[0:2]
#             alpha = re.findall(rf"{pattern}", response)
#             search_province_dict.append([self.get_district(province), province, alpha[0]])
#         return search_province_dict
#
#     def fake_form_data(self, searchKey, pro_alpha):
#         filter = "{\"r\":[{\"pr\":\"" + f"{pro_alpha}" + "\"}]}"
#         return json.dumps({
#             # return {
#             "filter": filter,
#             "pageIndex": '1',
#             "pageSize": '20',
#             "searchKey": f"{searchKey}",
#             # }
#         })
#
#     def get_pinyin(self, word, upper_first_char=False):
#         '''
#         汉字转拼音
#         :param word: 待转换word
#         :param upper_first_char: 拼音第一个字符是否大写
#         :return:
#         '''
#         s = ''
#         for i in pypinyin.lazy_pinyin(word, style=pypinyin.NORMAL):
#             if upper_first_char:
#                 s += ''.join(i.capitalize())
#             else:
#                 s += ''.join(i)
#         return s
#
#     def handle_search(self, response):
#         # q = re.findall(r"QCCSESSID=(.*?);", response.headers['Set-Cookie'])[0]
#         item = HangYeCompanyCount()
#         try:
#             citys = json.loads(response.text)['GroupItems'][3]['items']
#             for city_obj in citys:
#                 city = city_obj['desc']
#                 count = city_obj['count']
#                 item['district'] = response.meta['district']
#                 item['province'] = response.meta['province']
#                 item['city'] = city
#                 sort = response.meta['sort']
#                 key = self.get_pinyin(sort)
#                 item[key] = count
#                 yield item
#         except Exception as e:
#             print(e)
#             print('error========================')
#
#     def start_requests(self):
#         list_item = [
#             # ['消防', 'https://www.qcc.com/web/search?key=%E6%B6%88%E9%98%B2'],
#             # ['幕墙', 'https://www.qcc.com/web/search?key=%E5%B9%95%E5%A2%99'],
#             # ['钢构', 'https://www.qcc.com/web/search?key=%E9%92%A2%E6%9E%84'],
#             # ["一建", 'https://www.qcc.com/web/search?key=一建'],
#             # ["二建", 'https://www.qcc.com/web/search?key=二建'],
#             # ["三建", 'https://www.qcc.com/web/search?key=三建'],
#             # ["四建", 'https://www.qcc.com/web/search?key=四建'],
#             # ["五建", 'https://www.qcc.com/web/search?key=五建'],
#             # ["六建", 'https://www.qcc.com/web/search?key=六建'],
#             # ["七建", 'https://www.qcc.com/web/search?key=七建'],
#             # ["八建", 'https://www.qcc.com/web/search?key=八建'],
#             # ["九建", 'https://www.qcc.com/web/search?key=九建'],
#             # ["建筑工程", 'https://www.qcc.com/web/search?key=建筑工程'],
#             ["消防工程", 'https://www.qcc.com/web/search?key=消防工程'],
#             ["装饰工程", 'https://www.qcc.com/web/search?key=装饰工程'],
#             ["建设工程", 'https://www.qcc.com/web/search?key=建设工程'],
#             ["机电设备", 'https://www.qcc.com/web/search?key=机电设备'],
#             # ["机电工程", 'https://www.qcc.com/web/search?key=机电工程'],
#             # ["建筑安装", 'https://www.qcc.com/web/search?key=建筑安装'],
#             # ["机械设备", 'https://www.qcc.com/web/search?key=机械设备'],
#             # ["涂装", 'https://www.qcc.com/web/search?key=涂装'],
#             # ["钢结构", 'https://www.qcc.com/web/search?key=钢结构'],
#             # ["建筑材料", 'https://www.qcc.com/web/search?key=建筑材料'],
#             # ["通风设备", 'https://www.qcc.com/web/search?key=通风设备'],
#             # ["设备租赁", 'https://www.qcc.com/web/search?key=设备租赁'],
#             # ["机械租赁", 'https://www.qcc.com/web/search?key=机械租赁'],
#             # ["消防设备", 'https://www.qcc.com/web/search?key=消防设备'],
#             # ["防腐工程", 'https://www.qcc.com/web/search?key=防腐工程'],
#             # ["建筑设备", 'https://www.qcc.com/web/search?key=建筑设备'],
#             # ["起重机械", 'https://www.qcc.com/web/search?key=起重机械'],
#         ]
#         for item in list_item:
#             yield scrapy.Request(url=item[1], headers=self.fake_headers(),
#                                  callback=self.handle_response,
#                                  meta={
#                                      "sort": item[0]
#                                  })
#
#     def handle_response(self, response):
#         provinces = re.findall(r'(\w*)\(<span class="text-gray" data-v-8ad03c8e>', response.text)
#         if '总局' in provinces:
#             stop_index = provinces.index('总局')
#             provinces = provinces[:stop_index]
#         search_province_condition = self.get_search_condition(provinces, response.text)
#         q = 'eu9pahaigs5ai5d6ti9u295124'
#         for objs in search_province_condition:
#             headers = self.fake_json_requests_headers(q=q)
#             formdata = self.fake_form_data(pro_alpha=objs[2], searchKey=response.meta['sort'])
#             url = "https://www.qcc.com/api/search/searchMulti"
#             yield scrapy.FormRequest(
#                 url=url,
#                 headers=headers,
#                 method='post',
#                 body=formdata,
#                 callback=self.handle_search,
#                 dont_filter=True,
#                 meta={
#                     'district': objs[0],
#                     'province': objs[1],
#                     'sort': response.meta['sort']
#                 })
